{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import statsmodels\n",
    "import statsmodels.api as sm\n",
    "import statsmodels.formula.api as smf\n",
    "from scipy import stats\n",
    "import warnings;\n",
    "from pysqldf import SQLDF\n",
    "import pandasql as psql\n",
    "from matplotlib.ticker import FuncFormatter\n",
    "from sklearn.model_selection import KFold\n",
    "import sklearn.ensemble as ske\n",
    "import lightgbm as lgb\n",
    "from pandas.api.types import is_string_dtype\n",
    "from pandas.api.types import is_numeric_dtype\n",
    "import xgboost as xgb\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from math import sqrt\n",
    "\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "train01 = pd.read_csv(\"C:\\\\Kaggle\\\\Cars\\\\Data\\\\TrnDataForLGB.csv\")\n",
    "test01 = pd.read_csv(\"C:\\\\Kaggle\\\\Cars\\\\Data\\\\TstDataForLGB.csv\")\n",
    "\n",
    "Remove_List = [\"id\",\"Price\",\"Name\",\"Lag_Price2_MIN\",\"Lag_Price2_MAX\",\n",
    "               \"Lag_Price3_MIN\",\"Lag_Price3_MAX\",\"Lag_Price\",\"Lag_Price3\",\"Engine_Group\",\n",
    "               \"Power_Group\",\"TrainTestInd\",\"CarCompName\",\"RateChng1\",\"RateChng2\",\"RateChng3\",\n",
    "               \"Lag_Price4_MIN\",\"Lag_Price4_MAX\",\"Lag_Price4_MIN_BY_MAX\"]\n",
    "feature_names = list(set(list(train01.columns)) - set(Remove_List))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['New_Price', 'Lag_Price4', 'Lag_Price2'], dtype='object')"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train01[feature_names].columns[train01[feature_names].isnull().any()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['New_Price', 'Lag_Price4', 'Lag_Price2'], dtype='object')"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test01[feature_names].columns[test01[feature_names].isnull().any()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "train01['Lag_Price4'] = train01['Lag_Price4'].fillna(value=0)\n",
    "train01['Lag_Price2'] = train01['Lag_Price2'].fillna(value=0)\n",
    "train01['New_Price'] = train01['New_Price'].fillna(value=0)\n",
    "\n",
    "test01['Lag_Price4'] = test01['Lag_Price4'].fillna(value=0)\n",
    "test01['Lag_Price2'] = test01['Lag_Price2'].fillna(value=0)\n",
    "test01['New_Price'] = test01['New_Price'].fillna(value=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index([], dtype='object')\n",
      "Index([], dtype='object')\n"
     ]
    }
   ],
   "source": [
    "print(train01[feature_names].columns[train01[feature_names].isnull().any()])\n",
    "print(test01[feature_names].columns[test01[feature_names].isnull().any()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train02 = train01[feature_names]\n",
    "y_train02 = train01['Price']\n",
    "x_sub2 = test01[feature_names]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>min</th>\n",
       "      <th>25%</th>\n",
       "      <th>50%</th>\n",
       "      <th>75%</th>\n",
       "      <th>max</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Power</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>112.668894</td>\n",
       "      <td>53.940547</td>\n",
       "      <td>34.2</td>\n",
       "      <td>74.00</td>\n",
       "      <td>93.70</td>\n",
       "      <td>138.1</td>\n",
       "      <td>560.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LocationChennai</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.082073</td>\n",
       "      <td>0.274499</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Owner_TypeFourth...Above</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.001495</td>\n",
       "      <td>0.038643</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_CAMRY</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.001828</td>\n",
       "      <td>0.042714</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompName_NISSAN</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.015119</td>\n",
       "      <td>0.122036</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompName_BMW</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.044360</td>\n",
       "      <td>0.205910</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_ALTO</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.023924</td>\n",
       "      <td>0.152826</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_CLA</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.052500</td>\n",
       "      <td>0.223052</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_SCORPIO</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.010135</td>\n",
       "      <td>0.100168</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_TERRANO</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.004320</td>\n",
       "      <td>0.065587</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TransmissionManual</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.714238</td>\n",
       "      <td>0.451814</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LocationCoimbatore</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.105665</td>\n",
       "      <td>0.307434</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Kilometers_Driven</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>58738.380296</td>\n",
       "      <td>91268.843206</td>\n",
       "      <td>171.0</td>\n",
       "      <td>34000.00</td>\n",
       "      <td>53000.00</td>\n",
       "      <td>73000.0</td>\n",
       "      <td>6500000.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompName_MAHINDRA</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.045190</td>\n",
       "      <td>0.207738</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Mileage</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>17.796986</td>\n",
       "      <td>19.097099</td>\n",
       "      <td>-999.0</td>\n",
       "      <td>15.16</td>\n",
       "      <td>18.15</td>\n",
       "      <td>21.1</td>\n",
       "      <td>33.54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_GRAND</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.027081</td>\n",
       "      <td>0.162333</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_MOBILIO</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.002658</td>\n",
       "      <td>0.051494</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_BRIO</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.010135</td>\n",
       "      <td>0.100168</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_SUNNY</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.004320</td>\n",
       "      <td>0.065587</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_VENTO</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.017777</td>\n",
       "      <td>0.132151</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_DUSTER</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.013790</td>\n",
       "      <td>0.116627</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_COOPER</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.004320</td>\n",
       "      <td>0.065587</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>New_Price</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>3.600075</td>\n",
       "      <td>12.827771</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>230.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompName_FORD</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.049842</td>\n",
       "      <td>0.217637</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TransmissionAutomatic</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.285762</td>\n",
       "      <td>0.451814</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LocationMumbai</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.131251</td>\n",
       "      <td>0.337703</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompName_RENAULT</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.024423</td>\n",
       "      <td>0.154370</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_5</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.168134</td>\n",
       "      <td>0.374017</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_SANTA</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.002824</td>\n",
       "      <td>0.053074</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_X1</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.005316</td>\n",
       "      <td>0.072726</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_OMNI</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.003323</td>\n",
       "      <td>0.057553</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Fuel_TypeLPG</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.001661</td>\n",
       "      <td>0.040730</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompName_MERCEDES-BENZ</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.052833</td>\n",
       "      <td>0.223718</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Owner_TypeSecond</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.160824</td>\n",
       "      <td>0.367399</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_CIVIC</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.005316</td>\n",
       "      <td>0.072726</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LocationAhmedabad</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.037215</td>\n",
       "      <td>0.189305</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_COMPASS</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.002492</td>\n",
       "      <td>0.049863</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_ELITE</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.002492</td>\n",
       "      <td>0.049863</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_FIGO</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.016780</td>\n",
       "      <td>0.128458</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_OPTRA</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.001994</td>\n",
       "      <td>0.044610</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_X3</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.002326</td>\n",
       "      <td>0.048176</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LocationKolkata</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.088885</td>\n",
       "      <td>0.284602</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_JETTA</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.003987</td>\n",
       "      <td>0.063025</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_NANO</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.004154</td>\n",
       "      <td>0.064319</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Engine</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>1620.594949</td>\n",
       "      <td>601.030112</td>\n",
       "      <td>72.0</td>\n",
       "      <td>1198.00</td>\n",
       "      <td>1493.00</td>\n",
       "      <td>1984.0</td>\n",
       "      <td>5998.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_A-STAR</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.002824</td>\n",
       "      <td>0.053074</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_INDICA</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.006646</td>\n",
       "      <td>0.081256</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LocationBangalore</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.059478</td>\n",
       "      <td>0.236537</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_B</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.239741</td>\n",
       "      <td>0.426961</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_CRETA</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.015451</td>\n",
       "      <td>0.123349</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Fuel_TypeElectric</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.000332</td>\n",
       "      <td>0.018227</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_EECO</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.002991</td>\n",
       "      <td>0.054608</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_ETIOS</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.010301</td>\n",
       "      <td>0.100977</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_I20</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.043363</td>\n",
       "      <td>0.203689</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_M-CLASS</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.003821</td>\n",
       "      <td>0.061703</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_ENDEAVOUR</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.006479</td>\n",
       "      <td>0.080241</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_I10</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.051171</td>\n",
       "      <td>0.220365</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LocationHyderabad</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.123276</td>\n",
       "      <td>0.328781</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_S</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.689649</td>\n",
       "      <td>0.462676</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.00</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CompNameCarName_7</th>\n",
       "      <td>6019.0</td>\n",
       "      <td>0.044027</td>\n",
       "      <td>0.205173</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>157 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                            count          mean           std    min  \\\n",
       "Power                      6019.0    112.668894     53.940547   34.2   \n",
       "LocationChennai            6019.0      0.082073      0.274499    0.0   \n",
       "Owner_TypeFourth...Above   6019.0      0.001495      0.038643    0.0   \n",
       "CompNameCarName_CAMRY      6019.0      0.001828      0.042714    0.0   \n",
       "CompName_NISSAN            6019.0      0.015119      0.122036    0.0   \n",
       "CompName_BMW               6019.0      0.044360      0.205910    0.0   \n",
       "CompNameCarName_ALTO       6019.0      0.023924      0.152826    0.0   \n",
       "CompNameCarName_CLA        6019.0      0.052500      0.223052    0.0   \n",
       "CompNameCarName_SCORPIO    6019.0      0.010135      0.100168    0.0   \n",
       "CompNameCarName_TERRANO    6019.0      0.004320      0.065587    0.0   \n",
       "TransmissionManual         6019.0      0.714238      0.451814    0.0   \n",
       "LocationCoimbatore         6019.0      0.105665      0.307434    0.0   \n",
       "Kilometers_Driven          6019.0  58738.380296  91268.843206  171.0   \n",
       "CompName_MAHINDRA          6019.0      0.045190      0.207738    0.0   \n",
       "Mileage                    6019.0     17.796986     19.097099 -999.0   \n",
       "CompNameCarName_GRAND      6019.0      0.027081      0.162333    0.0   \n",
       "CompNameCarName_MOBILIO    6019.0      0.002658      0.051494    0.0   \n",
       "CompNameCarName_BRIO       6019.0      0.010135      0.100168    0.0   \n",
       "CompNameCarName_SUNNY      6019.0      0.004320      0.065587    0.0   \n",
       "CompNameCarName_VENTO      6019.0      0.017777      0.132151    0.0   \n",
       "CompNameCarName_DUSTER     6019.0      0.013790      0.116627    0.0   \n",
       "CompNameCarName_COOPER     6019.0      0.004320      0.065587    0.0   \n",
       "New_Price                  6019.0      3.600075     12.827771    0.0   \n",
       "CompName_FORD              6019.0      0.049842      0.217637    0.0   \n",
       "TransmissionAutomatic      6019.0      0.285762      0.451814    0.0   \n",
       "LocationMumbai             6019.0      0.131251      0.337703    0.0   \n",
       "CompName_RENAULT           6019.0      0.024423      0.154370    0.0   \n",
       "CompNameCarName_5          6019.0      0.168134      0.374017    0.0   \n",
       "CompNameCarName_SANTA      6019.0      0.002824      0.053074    0.0   \n",
       "CompNameCarName_X1         6019.0      0.005316      0.072726    0.0   \n",
       "...                           ...           ...           ...    ...   \n",
       "CompNameCarName_OMNI       6019.0      0.003323      0.057553    0.0   \n",
       "Fuel_TypeLPG               6019.0      0.001661      0.040730    0.0   \n",
       "CompName_MERCEDES-BENZ     6019.0      0.052833      0.223718    0.0   \n",
       "Owner_TypeSecond           6019.0      0.160824      0.367399    0.0   \n",
       "CompNameCarName_CIVIC      6019.0      0.005316      0.072726    0.0   \n",
       "LocationAhmedabad          6019.0      0.037215      0.189305    0.0   \n",
       "CompNameCarName_COMPASS    6019.0      0.002492      0.049863    0.0   \n",
       "CompNameCarName_ELITE      6019.0      0.002492      0.049863    0.0   \n",
       "CompNameCarName_FIGO       6019.0      0.016780      0.128458    0.0   \n",
       "CompNameCarName_OPTRA      6019.0      0.001994      0.044610    0.0   \n",
       "CompNameCarName_X3         6019.0      0.002326      0.048176    0.0   \n",
       "LocationKolkata            6019.0      0.088885      0.284602    0.0   \n",
       "CompNameCarName_JETTA      6019.0      0.003987      0.063025    0.0   \n",
       "CompNameCarName_NANO       6019.0      0.004154      0.064319    0.0   \n",
       "Engine                     6019.0   1620.594949    601.030112   72.0   \n",
       "CompNameCarName_A-STAR     6019.0      0.002824      0.053074    0.0   \n",
       "CompNameCarName_INDICA     6019.0      0.006646      0.081256    0.0   \n",
       "LocationBangalore          6019.0      0.059478      0.236537    0.0   \n",
       "CompNameCarName_B          6019.0      0.239741      0.426961    0.0   \n",
       "CompNameCarName_CRETA      6019.0      0.015451      0.123349    0.0   \n",
       "Fuel_TypeElectric          6019.0      0.000332      0.018227    0.0   \n",
       "CompNameCarName_EECO       6019.0      0.002991      0.054608    0.0   \n",
       "CompNameCarName_ETIOS      6019.0      0.010301      0.100977    0.0   \n",
       "CompNameCarName_I20        6019.0      0.043363      0.203689    0.0   \n",
       "CompNameCarName_M-CLASS    6019.0      0.003821      0.061703    0.0   \n",
       "CompNameCarName_ENDEAVOUR  6019.0      0.006479      0.080241    0.0   \n",
       "CompNameCarName_I10        6019.0      0.051171      0.220365    0.0   \n",
       "LocationHyderabad          6019.0      0.123276      0.328781    0.0   \n",
       "CompNameCarName_S          6019.0      0.689649      0.462676    0.0   \n",
       "CompNameCarName_7          6019.0      0.044027      0.205173    0.0   \n",
       "\n",
       "                                25%       50%      75%         max  \n",
       "Power                         74.00     93.70    138.1      560.00  \n",
       "LocationChennai                0.00      0.00      0.0        1.00  \n",
       "Owner_TypeFourth...Above       0.00      0.00      0.0        1.00  \n",
       "CompNameCarName_CAMRY          0.00      0.00      0.0        1.00  \n",
       "CompName_NISSAN                0.00      0.00      0.0        1.00  \n",
       "CompName_BMW                   0.00      0.00      0.0        1.00  \n",
       "CompNameCarName_ALTO           0.00      0.00      0.0        1.00  \n",
       "CompNameCarName_CLA            0.00      0.00      0.0        1.00  \n",
       "CompNameCarName_SCORPIO        0.00      0.00      0.0        1.00  \n",
       "CompNameCarName_TERRANO        0.00      0.00      0.0        1.00  \n",
       "TransmissionManual             0.00      1.00      1.0        1.00  \n",
       "LocationCoimbatore             0.00      0.00      0.0        1.00  \n",
       "Kilometers_Driven          34000.00  53000.00  73000.0  6500000.00  \n",
       "CompName_MAHINDRA              0.00      0.00      0.0        1.00  \n",
       "Mileage                       15.16     18.15     21.1       33.54  \n",
       "CompNameCarName_GRAND          0.00      0.00      0.0        1.00  \n",
       "CompNameCarName_MOBILIO        0.00      0.00      0.0        1.00  \n",
       "CompNameCarName_BRIO           0.00      0.00      0.0        1.00  \n",
       "CompNameCarName_SUNNY          0.00      0.00      0.0        1.00  \n",
       "CompNameCarName_VENTO          0.00      0.00      0.0        1.00  \n",
       "CompNameCarName_DUSTER         0.00      0.00      0.0        1.00  \n",
       "CompNameCarName_COOPER         0.00      0.00      0.0        1.00  \n",
       "New_Price                      0.00      0.00      0.0      230.00  \n",
       "CompName_FORD                  0.00      0.00      0.0        1.00  \n",
       "TransmissionAutomatic          0.00      0.00      1.0        1.00  \n",
       "LocationMumbai                 0.00      0.00      0.0        1.00  \n",
       "CompName_RENAULT               0.00      0.00      0.0        1.00  \n",
       "CompNameCarName_5              0.00      0.00      0.0        1.00  \n",
       "CompNameCarName_SANTA          0.00      0.00      0.0        1.00  \n",
       "CompNameCarName_X1             0.00      0.00      0.0        1.00  \n",
       "...                             ...       ...      ...         ...  \n",
       "CompNameCarName_OMNI           0.00      0.00      0.0        1.00  \n",
       "Fuel_TypeLPG                   0.00      0.00      0.0        1.00  \n",
       "CompName_MERCEDES-BENZ         0.00      0.00      0.0        1.00  \n",
       "Owner_TypeSecond               0.00      0.00      0.0        1.00  \n",
       "CompNameCarName_CIVIC          0.00      0.00      0.0        1.00  \n",
       "LocationAhmedabad              0.00      0.00      0.0        1.00  \n",
       "CompNameCarName_COMPASS        0.00      0.00      0.0        1.00  \n",
       "CompNameCarName_ELITE          0.00      0.00      0.0        1.00  \n",
       "CompNameCarName_FIGO           0.00      0.00      0.0        1.00  \n",
       "CompNameCarName_OPTRA          0.00      0.00      0.0        1.00  \n",
       "CompNameCarName_X3             0.00      0.00      0.0        1.00  \n",
       "LocationKolkata                0.00      0.00      0.0        1.00  \n",
       "CompNameCarName_JETTA          0.00      0.00      0.0        1.00  \n",
       "CompNameCarName_NANO           0.00      0.00      0.0        1.00  \n",
       "Engine                      1198.00   1493.00   1984.0     5998.00  \n",
       "CompNameCarName_A-STAR         0.00      0.00      0.0        1.00  \n",
       "CompNameCarName_INDICA         0.00      0.00      0.0        1.00  \n",
       "LocationBangalore              0.00      0.00      0.0        1.00  \n",
       "CompNameCarName_B              0.00      0.00      0.0        1.00  \n",
       "CompNameCarName_CRETA          0.00      0.00      0.0        1.00  \n",
       "Fuel_TypeElectric              0.00      0.00      0.0        1.00  \n",
       "CompNameCarName_EECO           0.00      0.00      0.0        1.00  \n",
       "CompNameCarName_ETIOS          0.00      0.00      0.0        1.00  \n",
       "CompNameCarName_I20            0.00      0.00      0.0        1.00  \n",
       "CompNameCarName_M-CLASS        0.00      0.00      0.0        1.00  \n",
       "CompNameCarName_ENDEAVOUR      0.00      0.00      0.0        1.00  \n",
       "CompNameCarName_I10            0.00      0.00      0.0        1.00  \n",
       "LocationHyderabad              0.00      0.00      0.0        1.00  \n",
       "CompNameCarName_S              0.00      1.00      1.0        1.00  \n",
       "CompNameCarName_7              0.00      0.00      0.0        1.00  \n",
       "\n",
       "[157 rows x 8 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_stats = x_train02.describe()\n",
    "train_stats = train_stats.transpose()\n",
    "train_stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def norm(x):\n",
    "  return(x - train_stats['mean']) / train_stats['std']\n",
    "normed_x_train02 = norm(x_train02)\n",
    "normed_x_sub2 = norm(x_sub2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "normed_x_train02.reset_index(drop = True, inplace = True)\n",
    "kf = KFold(n_splits = 5, shuffle = True, random_state = 100)\n",
    "kf.get_n_splits(normed_x_train02)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Activation\n",
    "from keras.layers.advanced_activations import PReLU\n",
    "from keras import optimizers\n",
    "from keras.callbacks import EarlyStopping\n",
    "import math\n",
    "\n",
    "def nn_model():\n",
    "    model = Sequential()\n",
    "    model.add(Dense(400, input_dim = normed_x_train02.shape[1], init = 'he_normal'))\n",
    "    model.add(Activation('sigmoid'))\n",
    "    model.add(Dropout(0.5))\n",
    "    model.add(Dense(400, init = 'he_normal'))\n",
    "    model.add(Activation('sigmoid'))\n",
    "    model.add(Dropout(0.5))\n",
    "    model.add(Dense(200, init = 'he_normal'))\n",
    "    model.add(Activation('sigmoid'))\n",
    "    model.add(Dropout(0.4))\n",
    "    model.add(Dense(1, init = 'he_normal'))\n",
    "    model.compile(loss = 'mean_squared_error', optimizer = 'Adam')\n",
    "    return(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running CV Iteration Num : 1\n",
      "Train on 4815 samples, validate on 1204 samples\n",
      "Epoch 1/200\n",
      "4815/4815 [==============================] - 4s 787us/step - loss: 0.9497 - val_loss: 0.0978\n",
      "Epoch 2/200\n",
      "4815/4815 [==============================] - 2s 361us/step - loss: 0.2295 - val_loss: 0.0567\n",
      "Epoch 3/200\n",
      "4815/4815 [==============================] - 2s 378us/step - loss: 0.1588 - val_loss: 0.0467\n",
      "Epoch 4/200\n",
      "4815/4815 [==============================] - 2s 372us/step - loss: 0.1344 - val_loss: 0.0384\n",
      "Epoch 5/200\n",
      "4815/4815 [==============================] - ETA: 0s - loss: 0.114 - 2s 353us/step - loss: 0.1145 - val_loss: 0.0359\n",
      "Epoch 6/200\n",
      "4815/4815 [==============================] - 2s 346us/step - loss: 0.1074 - val_loss: 0.0299\n",
      "Epoch 7/200\n",
      "4815/4815 [==============================] - 2s 351us/step - loss: 0.1005 - val_loss: 0.0292\n",
      "Epoch 8/200\n",
      "4815/4815 [==============================] - 2s 357us/step - loss: 0.0954 - val_loss: 0.0285\n",
      "Epoch 9/200\n",
      "4815/4815 [==============================] - 2s 352us/step - loss: 0.0892 - val_loss: 0.0342\n",
      "Epoch 10/200\n",
      "4815/4815 [==============================] - 2s 360us/step - loss: 0.0843 - val_loss: 0.0268\n",
      "Epoch 11/200\n",
      "4815/4815 [==============================] - 2s 351us/step - loss: 0.0840 - val_loss: 0.0253\n",
      "Epoch 12/200\n",
      "4815/4815 [==============================] - 2s 383us/step - loss: 0.0813 - val_loss: 0.0258\n",
      "Epoch 13/200\n",
      "4815/4815 [==============================] - 2s 365us/step - loss: 0.0787 - val_loss: 0.0409\n",
      "Epoch 14/200\n",
      "4815/4815 [==============================] - 2s 361us/step - loss: 0.0767 - val_loss: 0.0242\n",
      "Epoch 15/200\n",
      "4815/4815 [==============================] - 2s 341us/step - loss: 0.0716 - val_loss: 0.0245\n",
      "Epoch 16/200\n",
      "4815/4815 [==============================] - 2s 345us/step - loss: 0.0724 - val_loss: 0.0211\n",
      "Epoch 17/200\n",
      "4815/4815 [==============================] - 2s 358us/step - loss: 0.0706 - val_loss: 0.0361\n",
      "Epoch 18/200\n",
      "4815/4815 [==============================] - 2s 375us/step - loss: 0.0644 - val_loss: 0.0225\n",
      "Epoch 19/200\n",
      "4815/4815 [==============================] - 2s 369us/step - loss: 0.0657 - val_loss: 0.0222\n",
      "Epoch 20/200\n",
      "4815/4815 [==============================] - 2s 359us/step - loss: 0.0630 - val_loss: 0.0261\n",
      "Epoch 21/200\n",
      "4815/4815 [==============================] - 2s 374us/step - loss: 0.0630 - val_loss: 0.0215\n",
      "Epoch 22/200\n",
      "4815/4815 [==============================] - 2s 358us/step - loss: 0.0627 - val_loss: 0.0306\n",
      "Epoch 23/200\n",
      "4815/4815 [==============================] - 2s 343us/step - loss: 0.0584 - val_loss: 0.0218\n",
      "Epoch 24/200\n",
      "4815/4815 [==============================] - 2s 353us/step - loss: 0.0574 - val_loss: 0.0210\n",
      "Epoch 25/200\n",
      "4815/4815 [==============================] - ETA: 0s - loss: 0.057 - 2s 364us/step - loss: 0.0572 - val_loss: 0.0235\n",
      "Epoch 26/200\n",
      "4815/4815 [==============================] - 2s 360us/step - loss: 0.0563 - val_loss: 0.0198\n",
      "Epoch 27/200\n",
      "4815/4815 [==============================] - 2s 375us/step - loss: 0.0543 - val_loss: 0.0229\n",
      "Epoch 28/200\n",
      "4815/4815 [==============================] - 2s 371us/step - loss: 0.0530 - val_loss: 0.0235\n",
      "Epoch 29/200\n",
      "4815/4815 [==============================] - 2s 350us/step - loss: 0.0553 - val_loss: 0.0224\n",
      "Epoch 30/200\n",
      "4815/4815 [==============================] - 2s 351us/step - loss: 0.0512 - val_loss: 0.0198\n",
      "Epoch 31/200\n",
      "4815/4815 [==============================] - 2s 374us/step - loss: 0.0496 - val_loss: 0.0207\n",
      "Epoch 32/200\n",
      "4815/4815 [==============================] - 2s 370us/step - loss: 0.0496 - val_loss: 0.0204\n",
      "Epoch 33/200\n",
      "4815/4815 [==============================] - 2s 368us/step - loss: 0.0493 - val_loss: 0.0189\n",
      "Epoch 34/200\n",
      "4815/4815 [==============================] - 2s 352us/step - loss: 0.0469 - val_loss: 0.0201\n",
      "Epoch 35/200\n",
      "4815/4815 [==============================] - 2s 369us/step - loss: 0.0466 - val_loss: 0.0248\n",
      "Epoch 36/200\n",
      "4815/4815 [==============================] - 2s 379us/step - loss: 0.0459 - val_loss: 0.0201\n",
      "Epoch 37/200\n",
      "4815/4815 [==============================] - 2s 368us/step - loss: 0.0443 - val_loss: 0.0207\n",
      "Epoch 38/200\n",
      "4815/4815 [==============================] - 2s 352us/step - loss: 0.0445 - val_loss: 0.0209\n",
      "Epoch 39/200\n",
      "4815/4815 [==============================] - 2s 381us/step - loss: 0.0428 - val_loss: 0.0231\n",
      "Epoch 40/200\n",
      "4815/4815 [==============================] - 2s 357us/step - loss: 0.0428 - val_loss: 0.0208\n",
      "Epoch 41/200\n",
      "4815/4815 [==============================] - 2s 348us/step - loss: 0.0428 - val_loss: 0.0219\n",
      "Epoch 42/200\n",
      "4815/4815 [==============================] - 2s 374us/step - loss: 0.0413 - val_loss: 0.0226\n",
      "Epoch 43/200\n",
      "4815/4815 [==============================] - 2s 351us/step - loss: 0.0403 - val_loss: 0.0194\n",
      "Epoch 00043: early stopping\n",
      "Test RMSE :  0.13943182078875174\n",
      "Running CV Iteration Num : 2\n",
      "Train on 4815 samples, validate on 1204 samples\n",
      "Epoch 1/200\n",
      "4815/4815 [==============================] - 3s 724us/step - loss: 0.7727 - val_loss: 0.1075\n",
      "Epoch 2/200\n",
      "4815/4815 [==============================] - 2s 403us/step - loss: 0.1940 - val_loss: 0.0533\n",
      "Epoch 3/200\n",
      "4815/4815 [==============================] - 2s 350us/step - loss: 0.1369 - val_loss: 0.0541\n",
      "Epoch 4/200\n",
      "4815/4815 [==============================] - 2s 356us/step - loss: 0.1176 - val_loss: 0.0400\n",
      "Epoch 5/200\n",
      "4815/4815 [==============================] - 2s 351us/step - loss: 0.1049 - val_loss: 0.0383\n",
      "Epoch 6/200\n",
      "4815/4815 [==============================] - 2s 356us/step - loss: 0.0935 - val_loss: 0.0345\n",
      "Epoch 7/200\n",
      "4815/4815 [==============================] - 2s 401us/step - loss: 0.0907 - val_loss: 0.0322\n",
      "Epoch 8/200\n",
      "4815/4815 [==============================] - 2s 412us/step - loss: 0.0844 - val_loss: 0.0301\n",
      "Epoch 9/200\n",
      "4815/4815 [==============================] - 2s 437us/step - loss: 0.0874 - val_loss: 0.0278\n",
      "Epoch 10/200\n",
      "4815/4815 [==============================] - 2s 385us/step - loss: 0.0800 - val_loss: 0.0269\n",
      "Epoch 11/200\n",
      "4815/4815 [==============================] - 2s 378us/step - loss: 0.0755 - val_loss: 0.0272\n",
      "Epoch 12/200\n",
      "4815/4815 [==============================] - 2s 382us/step - loss: 0.0721 - val_loss: 0.0276\n",
      "Epoch 13/200\n",
      "4815/4815 [==============================] - 2s 378us/step - loss: 0.0684 - val_loss: 0.0277\n",
      "Epoch 14/200\n",
      "4815/4815 [==============================] - 2s 378us/step - loss: 0.0700 - val_loss: 0.0433\n",
      "Epoch 15/200\n",
      "4815/4815 [==============================] - 2s 385us/step - loss: 0.0659 - val_loss: 0.0365\n",
      "Epoch 16/200\n",
      "4815/4815 [==============================] - 2s 383us/step - loss: 0.0657 - val_loss: 0.0338\n",
      "Epoch 17/200\n",
      "4815/4815 [==============================] - 2s 382us/step - loss: 0.0647 - val_loss: 0.0254\n",
      "Epoch 18/200\n",
      "4815/4815 [==============================] - 2s 375us/step - loss: 0.0601 - val_loss: 0.0291\n",
      "Epoch 19/200\n",
      "4815/4815 [==============================] - 2s 380us/step - loss: 0.0571 - val_loss: 0.0243\n",
      "Epoch 20/200\n",
      "4815/4815 [==============================] - 2s 380us/step - loss: 0.0583 - val_loss: 0.0284\n",
      "Epoch 21/200\n",
      "4815/4815 [==============================] - 2s 385us/step - loss: 0.0566 - val_loss: 0.0264\n",
      "Epoch 22/200\n",
      "4815/4815 [==============================] - 2s 377us/step - loss: 0.0556 - val_loss: 0.0329\n",
      "Epoch 23/200\n",
      "4815/4815 [==============================] - 2s 377us/step - loss: 0.0548 - val_loss: 0.0295\n",
      "Epoch 24/200\n",
      "4815/4815 [==============================] - 2s 392us/step - loss: 0.0531 - val_loss: 0.0242\n",
      "Epoch 25/200\n",
      "4815/4815 [==============================] - 2s 375us/step - loss: 0.0509 - val_loss: 0.0257\n",
      "Epoch 26/200\n",
      "4815/4815 [==============================] - 2s 384us/step - loss: 0.0495 - val_loss: 0.0284\n",
      "Epoch 27/200\n",
      "4815/4815 [==============================] - 2s 386us/step - loss: 0.0498 - val_loss: 0.0248\n",
      "Epoch 28/200\n",
      "4815/4815 [==============================] - 2s 385us/step - loss: 0.0490 - val_loss: 0.0236\n",
      "Epoch 29/200\n",
      "4815/4815 [==============================] - 2s 393us/step - loss: 0.0497 - val_loss: 0.0234\n",
      "Epoch 30/200\n",
      "4815/4815 [==============================] - 2s 389us/step - loss: 0.0467 - val_loss: 0.0305\n",
      "Epoch 31/200\n",
      "4815/4815 [==============================] - 2s 392us/step - loss: 0.0451 - val_loss: 0.0244\n",
      "Epoch 32/200\n",
      "4815/4815 [==============================] - 2s 382us/step - loss: 0.0448 - val_loss: 0.0238\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 33/200\n",
      "4815/4815 [==============================] - 2s 353us/step - loss: 0.0448 - val_loss: 0.0243\n",
      "Epoch 34/200\n",
      "4815/4815 [==============================] - 2s 357us/step - loss: 0.0441 - val_loss: 0.0288\n",
      "Epoch 35/200\n",
      "4815/4815 [==============================] - 2s 359us/step - loss: 0.0441 - val_loss: 0.0245\n",
      "Epoch 36/200\n",
      "4815/4815 [==============================] - 2s 362us/step - loss: 0.0425 - val_loss: 0.0302\n",
      "Epoch 37/200\n",
      "4815/4815 [==============================] - 2s 357us/step - loss: 0.0407 - val_loss: 0.0308\n",
      "Epoch 38/200\n",
      "4815/4815 [==============================] - 2s 355us/step - loss: 0.0405 - val_loss: 0.0245\n",
      "Epoch 39/200\n",
      "4815/4815 [==============================] - 2s 360us/step - loss: 0.0395 - val_loss: 0.0236\n",
      "Epoch 00039: early stopping\n",
      "Test RMSE :  0.15371392636132974\n",
      "Running CV Iteration Num : 3\n",
      "Train on 4815 samples, validate on 1204 samples\n",
      "Epoch 1/200\n",
      "4815/4815 [==============================] - 4s 778us/step - loss: 0.8769 - val_loss: 0.0946\n",
      "Epoch 2/200\n",
      "4815/4815 [==============================] - 2s 380us/step - loss: 0.2116 - val_loss: 0.0652\n",
      "Epoch 3/200\n",
      "4815/4815 [==============================] - 2s 384us/step - loss: 0.1446 - val_loss: 0.0583\n",
      "Epoch 4/200\n",
      "4815/4815 [==============================] - 2s 376us/step - loss: 0.1251 - val_loss: 0.0618\n",
      "Epoch 5/200\n",
      "4815/4815 [==============================] - 2s 376us/step - loss: 0.1108 - val_loss: 0.0478\n",
      "Epoch 6/200\n",
      "4815/4815 [==============================] - 2s 384us/step - loss: 0.0973 - val_loss: 0.0461\n",
      "Epoch 7/200\n",
      "4815/4815 [==============================] - 2s 389us/step - loss: 0.0936 - val_loss: 0.0455\n",
      "Epoch 8/200\n",
      "4815/4815 [==============================] - 2s 380us/step - loss: 0.0867 - val_loss: 0.0442\n",
      "Epoch 9/200\n",
      "4815/4815 [==============================] - 2s 379us/step - loss: 0.0802 - val_loss: 0.0416\n",
      "Epoch 10/200\n",
      "4815/4815 [==============================] - 2s 377us/step - loss: 0.0799 - val_loss: 0.0475\n",
      "Epoch 11/200\n",
      "4815/4815 [==============================] - 2s 382us/step - loss: 0.0756 - val_loss: 0.0430\n",
      "Epoch 12/200\n",
      "4815/4815 [==============================] - 2s 384us/step - loss: 0.0748 - val_loss: 0.0390\n",
      "Epoch 13/200\n",
      "4815/4815 [==============================] - 2s 376us/step - loss: 0.0700 - val_loss: 0.0414\n",
      "Epoch 14/200\n",
      "4815/4815 [==============================] - 2s 378us/step - loss: 0.0674 - val_loss: 0.0439\n",
      "Epoch 15/200\n",
      "4815/4815 [==============================] - 2s 389us/step - loss: 0.0654 - val_loss: 0.0395\n",
      "Epoch 16/200\n",
      "4815/4815 [==============================] - 2s 380us/step - loss: 0.0641 - val_loss: 0.0408\n",
      "Epoch 17/200\n",
      "4815/4815 [==============================] - 2s 388us/step - loss: 0.0638 - val_loss: 0.0383\n",
      "Epoch 18/200\n",
      "4815/4815 [==============================] - 2s 381us/step - loss: 0.0600 - val_loss: 0.0379\n",
      "Epoch 19/200\n",
      "4815/4815 [==============================] - 2s 380us/step - loss: 0.0611 - val_loss: 0.0369\n",
      "Epoch 20/200\n",
      "4815/4815 [==============================] - 2s 384us/step - loss: 0.0599 - val_loss: 0.0497\n",
      "Epoch 21/200\n",
      "4815/4815 [==============================] - 2s 386us/step - loss: 0.0590 - val_loss: 0.0370\n",
      "Epoch 22/200\n",
      "4815/4815 [==============================] - 2s 391us/step - loss: 0.0538 - val_loss: 0.0382\n",
      "Epoch 23/200\n",
      "4815/4815 [==============================] - 2s 383us/step - loss: 0.0543 - val_loss: 0.0366\n",
      "Epoch 24/200\n",
      "4815/4815 [==============================] - 2s 387us/step - loss: 0.0544 - val_loss: 0.0388\n",
      "Epoch 25/200\n",
      "4815/4815 [==============================] - 2s 387us/step - loss: 0.0531 - val_loss: 0.0456\n",
      "Epoch 26/200\n",
      "4815/4815 [==============================] - 2s 379us/step - loss: 0.0534 - val_loss: 0.0371\n",
      "Epoch 27/200\n",
      "4815/4815 [==============================] - 2s 380us/step - loss: 0.0504 - val_loss: 0.0361\n",
      "Epoch 28/200\n",
      "4815/4815 [==============================] - 2s 380us/step - loss: 0.0476 - val_loss: 0.0383\n",
      "Epoch 29/200\n",
      "4815/4815 [==============================] - 2s 371us/step - loss: 0.0484 - val_loss: 0.0375\n",
      "Epoch 30/200\n",
      "4815/4815 [==============================] - 2s 379us/step - loss: 0.0479 - val_loss: 0.0388\n",
      "Epoch 31/200\n",
      "4815/4815 [==============================] - 2s 403us/step - loss: 0.0460 - val_loss: 0.0376\n",
      "Epoch 32/200\n",
      "4815/4815 [==============================] - 2s 435us/step - loss: 0.0447 - val_loss: 0.0361\n",
      "Epoch 33/200\n",
      "4815/4815 [==============================] - 2s 388us/step - loss: 0.0427 - val_loss: 0.0359\n",
      "Epoch 34/200\n",
      "4815/4815 [==============================] - 2s 388us/step - loss: 0.0437 - val_loss: 0.0404\n",
      "Epoch 35/200\n",
      "4815/4815 [==============================] - 2s 387us/step - loss: 0.0458 - val_loss: 0.0346\n",
      "Epoch 36/200\n",
      "4815/4815 [==============================] - 2s 390us/step - loss: 0.0431 - val_loss: 0.0348\n",
      "Epoch 37/200\n",
      "4815/4815 [==============================] - 2s 382us/step - loss: 0.0411 - val_loss: 0.0364\n",
      "Epoch 38/200\n",
      "4815/4815 [==============================] - 2s 385us/step - loss: 0.0400 - val_loss: 0.0414\n",
      "Epoch 39/200\n",
      "4815/4815 [==============================] - 2s 390us/step - loss: 0.0385 - val_loss: 0.0359\n",
      "Epoch 40/200\n",
      "4815/4815 [==============================] - 2s 391us/step - loss: 0.0385 - val_loss: 0.0344\n",
      "Epoch 41/200\n",
      "4815/4815 [==============================] - 2s 383us/step - loss: 0.0377 - val_loss: 0.0348\n",
      "Epoch 42/200\n",
      "4815/4815 [==============================] - 2s 387us/step - loss: 0.0386 - val_loss: 0.0349\n",
      "Epoch 43/200\n",
      "4815/4815 [==============================] - 2s 385us/step - loss: 0.0370 - val_loss: 0.0358\n",
      "Epoch 44/200\n",
      "4815/4815 [==============================] - 2s 421us/step - loss: 0.0366 - val_loss: 0.0348\n",
      "Epoch 45/200\n",
      "4815/4815 [==============================] - 2s 390us/step - loss: 0.0362 - val_loss: 0.0365\n",
      "Epoch 46/200\n",
      "4815/4815 [==============================] - 2s 454us/step - loss: 0.0354 - val_loss: 0.0363\n",
      "Epoch 47/200\n",
      "4815/4815 [==============================] - 2s 441us/step - loss: 0.0348 - val_loss: 0.0352\n",
      "Epoch 48/200\n",
      "4815/4815 [==============================] - 2s 421us/step - loss: 0.0355 - val_loss: 0.0338\n",
      "Epoch 49/200\n",
      "4815/4815 [==============================] - 2s 379us/step - loss: 0.0336 - val_loss: 0.0337\n",
      "Epoch 50/200\n",
      "4815/4815 [==============================] - 2s 400us/step - loss: 0.0346 - val_loss: 0.0345\n",
      "Epoch 51/200\n",
      "4815/4815 [==============================] - 2s 393us/step - loss: 0.0343 - val_loss: 0.0408\n",
      "Epoch 52/200\n",
      "4815/4815 [==============================] - 2s 397us/step - loss: 0.0346 - val_loss: 0.0343\n",
      "Epoch 53/200\n",
      "4815/4815 [==============================] - 2s 392us/step - loss: 0.0330 - val_loss: 0.0336\n",
      "Epoch 54/200\n",
      "4815/4815 [==============================] - 2s 408us/step - loss: 0.0324 - val_loss: 0.0331\n",
      "Epoch 55/200\n",
      "4815/4815 [==============================] - 2s 394us/step - loss: 0.0328 - val_loss: 0.0351\n",
      "Epoch 56/200\n",
      "4815/4815 [==============================] - 2s 402us/step - loss: 0.0330 - val_loss: 0.0366\n",
      "Epoch 57/200\n",
      "4815/4815 [==============================] - 2s 369us/step - loss: 0.0306 - val_loss: 0.0340\n",
      "Epoch 58/200\n",
      "4815/4815 [==============================] - 2s 396us/step - loss: 0.0304 - val_loss: 0.0370\n",
      "Epoch 59/200\n",
      "4815/4815 [==============================] - 2s 427us/step - loss: 0.0305 - val_loss: 0.0331\n",
      "Epoch 60/200\n",
      "4815/4815 [==============================] - 2s 375us/step - loss: 0.0308 - val_loss: 0.0339\n",
      "Epoch 61/200\n",
      "4815/4815 [==============================] - 2s 386us/step - loss: 0.0299 - val_loss: 0.0334\n",
      "Epoch 62/200\n",
      "4815/4815 [==============================] - 2s 403us/step - loss: 0.0291 - val_loss: 0.0340\n",
      "Epoch 63/200\n",
      "4815/4815 [==============================] - 2s 401us/step - loss: 0.0295 - val_loss: 0.0332\n",
      "Epoch 64/200\n",
      "4815/4815 [==============================] - 2s 436us/step - loss: 0.0304 - val_loss: 0.0335\n",
      "Epoch 65/200\n",
      "4815/4815 [==============================] - 2s 398us/step - loss: 0.0283 - val_loss: 0.0331\n",
      "Epoch 66/200\n",
      "4815/4815 [==============================] - 2s 385us/step - loss: 0.0291 - val_loss: 0.0324\n",
      "Epoch 67/200\n",
      "4815/4815 [==============================] - 2s 400us/step - loss: 0.0278 - val_loss: 0.0341\n",
      "Epoch 68/200\n",
      "4815/4815 [==============================] - 2s 394us/step - loss: 0.0277 - val_loss: 0.0337\n",
      "Epoch 69/200\n",
      "4815/4815 [==============================] - 2s 397us/step - loss: 0.0277 - val_loss: 0.0329\n",
      "Epoch 70/200\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4815/4815 [==============================] - 2s 351us/step - loss: 0.0265 - val_loss: 0.0323\n",
      "Epoch 71/200\n",
      "4815/4815 [==============================] - 2s 361us/step - loss: 0.0273 - val_loss: 0.0367\n",
      "Epoch 72/200\n",
      "4815/4815 [==============================] - 2s 366us/step - loss: 0.0279 - val_loss: 0.0402\n",
      "Epoch 73/200\n",
      "4815/4815 [==============================] - 2s 375us/step - loss: 0.0282 - val_loss: 0.0344\n",
      "Epoch 74/200\n",
      "4815/4815 [==============================] - 2s 374us/step - loss: 0.0283 - val_loss: 0.0333\n",
      "Epoch 75/200\n",
      "4815/4815 [==============================] - 2s 367us/step - loss: 0.0264 - val_loss: 0.0336\n",
      "Epoch 76/200\n",
      "4815/4815 [==============================] - 2s 374us/step - loss: 0.0257 - val_loss: 0.0330\n",
      "Epoch 77/200\n",
      "4815/4815 [==============================] - 2s 372us/step - loss: 0.0258 - val_loss: 0.0339\n",
      "Epoch 78/200\n",
      "4815/4815 [==============================] - 2s 374us/step - loss: 0.0242 - val_loss: 0.0324\n",
      "Epoch 79/200\n",
      "4815/4815 [==============================] - 2s 376us/step - loss: 0.0253 - val_loss: 0.0336\n",
      "Epoch 80/200\n",
      "4815/4815 [==============================] - 2s 383us/step - loss: 0.0242 - val_loss: 0.0318\n",
      "Epoch 81/200\n",
      "4815/4815 [==============================] - 2s 391us/step - loss: 0.0259 - val_loss: 0.0334\n",
      "Epoch 82/200\n",
      "4815/4815 [==============================] - 2s 386us/step - loss: 0.0250 - val_loss: 0.0344\n",
      "Epoch 83/200\n",
      "4815/4815 [==============================] - 2s 375us/step - loss: 0.0251 - val_loss: 0.0351\n",
      "Epoch 84/200\n",
      "4815/4815 [==============================] - 2s 375us/step - loss: 0.0268 - val_loss: 0.0319\n",
      "Epoch 85/200\n",
      "4815/4815 [==============================] - 2s 380us/step - loss: 0.0244 - val_loss: 0.0332\n",
      "Epoch 86/200\n",
      "4815/4815 [==============================] - 2s 385us/step - loss: 0.0245 - val_loss: 0.0334\n",
      "Epoch 87/200\n",
      "4815/4815 [==============================] - 2s 380us/step - loss: 0.0241 - val_loss: 0.0322\n",
      "Epoch 88/200\n",
      "4815/4815 [==============================] - 2s 385us/step - loss: 0.0244 - val_loss: 0.0326\n",
      "Epoch 89/200\n",
      "4815/4815 [==============================] - 2s 376us/step - loss: 0.0244 - val_loss: 0.0320\n",
      "Epoch 90/200\n",
      "4815/4815 [==============================] - 2s 379us/step - loss: 0.0233 - val_loss: 0.0334\n",
      "Epoch 00090: early stopping\n",
      "Test RMSE :  0.18288052265929983\n",
      "Running CV Iteration Num : 4\n",
      "Train on 4815 samples, validate on 1204 samples\n",
      "Epoch 1/200\n",
      "4815/4815 [==============================] - 4s 811us/step - loss: 0.7842 - val_loss: 0.0852\n",
      "Epoch 2/200\n",
      "4815/4815 [==============================] - 2s 365us/step - loss: 0.2007 - val_loss: 0.0604\n",
      "Epoch 3/200\n",
      "4815/4815 [==============================] - 2s 389us/step - loss: 0.1407 - val_loss: 0.0461\n",
      "Epoch 4/200\n",
      "4815/4815 [==============================] - ETA: 0s - loss: 0.119 - 2s 373us/step - loss: 0.1197 - val_loss: 0.0377\n",
      "Epoch 5/200\n",
      "4815/4815 [==============================] - 2s 402us/step - loss: 0.1106 - val_loss: 0.0638\n",
      "Epoch 6/200\n",
      "4815/4815 [==============================] - 2s 382us/step - loss: 0.1002 - val_loss: 0.0382\n",
      "Epoch 7/200\n",
      "4815/4815 [==============================] - 2s 373us/step - loss: 0.0919 - val_loss: 0.0343\n",
      "Epoch 8/200\n",
      "4815/4815 [==============================] - 2s 375us/step - loss: 0.0910 - val_loss: 0.0538\n",
      "Epoch 9/200\n",
      "4815/4815 [==============================] - 2s 377us/step - loss: 0.0872 - val_loss: 0.0319\n",
      "Epoch 10/200\n",
      "4815/4815 [==============================] - 2s 374us/step - loss: 0.0825 - val_loss: 0.0315\n",
      "Epoch 11/200\n",
      "4815/4815 [==============================] - 2s 378us/step - loss: 0.0748 - val_loss: 0.0276\n",
      "Epoch 12/200\n",
      "4815/4815 [==============================] - 2s 376us/step - loss: 0.0769 - val_loss: 0.0324\n",
      "Epoch 13/200\n",
      "4815/4815 [==============================] - 2s 383us/step - loss: 0.0723 - val_loss: 0.0273\n",
      "Epoch 14/200\n",
      "4815/4815 [==============================] - 2s 380us/step - loss: 0.0726 - val_loss: 0.0286\n",
      "Epoch 15/200\n",
      "4815/4815 [==============================] - 2s 379us/step - loss: 0.0654 - val_loss: 0.0268\n",
      "Epoch 16/200\n",
      "4815/4815 [==============================] - 2s 375us/step - loss: 0.0676 - val_loss: 0.0305\n",
      "Epoch 17/200\n",
      "4815/4815 [==============================] - 2s 375us/step - loss: 0.0647 - val_loss: 0.0313\n",
      "Epoch 18/200\n",
      "4815/4815 [==============================] - 2s 373us/step - loss: 0.0626 - val_loss: 0.0276\n",
      "Epoch 19/200\n",
      "4815/4815 [==============================] - 2s 371us/step - loss: 0.0601 - val_loss: 0.0273\n",
      "Epoch 20/200\n",
      "4815/4815 [==============================] - 2s 375us/step - loss: 0.0595 - val_loss: 0.0257\n",
      "Epoch 21/200\n",
      "4815/4815 [==============================] - 2s 379us/step - loss: 0.0564 - val_loss: 0.0259\n",
      "Epoch 22/200\n",
      "4815/4815 [==============================] - 2s 376us/step - loss: 0.0567 - val_loss: 0.0251\n",
      "Epoch 23/200\n",
      "4815/4815 [==============================] - 2s 372us/step - loss: 0.0549 - val_loss: 0.0287\n",
      "Epoch 24/200\n",
      "4815/4815 [==============================] - 2s 384us/step - loss: 0.0536 - val_loss: 0.0254\n",
      "Epoch 25/200\n",
      "4815/4815 [==============================] - 2s 372us/step - loss: 0.0539 - val_loss: 0.0257\n",
      "Epoch 26/200\n",
      "4815/4815 [==============================] - 2s 377us/step - loss: 0.0529 - val_loss: 0.0253\n",
      "Epoch 27/200\n",
      "4815/4815 [==============================] - 2s 399us/step - loss: 0.0498 - val_loss: 0.0251\n",
      "Epoch 28/200\n",
      "4815/4815 [==============================] - 2s 392us/step - loss: 0.0514 - val_loss: 0.0253\n",
      "Epoch 29/200\n",
      "4815/4815 [==============================] - 2s 383us/step - loss: 0.0495 - val_loss: 0.0247\n",
      "Epoch 30/200\n",
      "4815/4815 [==============================] - 2s 374us/step - loss: 0.0485 - val_loss: 0.0280\n",
      "Epoch 31/200\n",
      "4815/4815 [==============================] - 2s 377us/step - loss: 0.0453 - val_loss: 0.0265\n",
      "Epoch 32/200\n",
      "4815/4815 [==============================] - 2s 378us/step - loss: 0.0466 - val_loss: 0.0242\n",
      "Epoch 33/200\n",
      "4815/4815 [==============================] - 2s 375us/step - loss: 0.0463 - val_loss: 0.0368\n",
      "Epoch 34/200\n",
      "4815/4815 [==============================] - 2s 380us/step - loss: 0.0449 - val_loss: 0.0240\n",
      "Epoch 35/200\n",
      "4815/4815 [==============================] - 2s 384us/step - loss: 0.0434 - val_loss: 0.0237\n",
      "Epoch 36/200\n",
      "4815/4815 [==============================] - 2s 380us/step - loss: 0.0452 - val_loss: 0.0241\n",
      "Epoch 37/200\n",
      "4815/4815 [==============================] - 2s 373us/step - loss: 0.0426 - val_loss: 0.0250\n",
      "Epoch 38/200\n",
      "4815/4815 [==============================] - 2s 425us/step - loss: 0.0425 - val_loss: 0.0245\n",
      "Epoch 39/200\n",
      "4815/4815 [==============================] - 2s 379us/step - loss: 0.0400 - val_loss: 0.0262\n",
      "Epoch 40/200\n",
      "4815/4815 [==============================] - 2s 380us/step - loss: 0.0391 - val_loss: 0.0239\n",
      "Epoch 41/200\n",
      "4815/4815 [==============================] - 2s 378us/step - loss: 0.0395 - val_loss: 0.0251\n",
      "Epoch 42/200\n",
      "4815/4815 [==============================] - 2s 376us/step - loss: 0.0388 - val_loss: 0.0241\n",
      "Epoch 43/200\n",
      "4815/4815 [==============================] - 2s 386us/step - loss: 0.0374 - val_loss: 0.0238\n",
      "Epoch 44/200\n",
      "4815/4815 [==============================] - 2s 379us/step - loss: 0.0380 - val_loss: 0.0236\n",
      "Epoch 45/200\n",
      "4815/4815 [==============================] - 2s 365us/step - loss: 0.0374 - val_loss: 0.0271\n",
      "Epoch 46/200\n",
      "4815/4815 [==============================] - 2s 384us/step - loss: 0.0370 - val_loss: 0.0265\n",
      "Epoch 47/200\n",
      "4815/4815 [==============================] - 2s 373us/step - loss: 0.0379 - val_loss: 0.0249\n",
      "Epoch 48/200\n",
      "4815/4815 [==============================] - 2s 389us/step - loss: 0.0352 - val_loss: 0.0242\n",
      "Epoch 49/200\n",
      "4815/4815 [==============================] - 2s 384us/step - loss: 0.0350 - val_loss: 0.0247\n",
      "Epoch 50/200\n",
      "4815/4815 [==============================] - 2s 381us/step - loss: 0.0345 - val_loss: 0.0257\n",
      "Epoch 51/200\n",
      "4815/4815 [==============================] - 2s 380us/step - loss: 0.0348 - val_loss: 0.0238\n",
      "Epoch 52/200\n",
      "4815/4815 [==============================] - 2s 384us/step - loss: 0.0339 - val_loss: 0.0235\n",
      "Epoch 53/200\n",
      "4815/4815 [==============================] - 2s 381us/step - loss: 0.0330 - val_loss: 0.0228\n",
      "Epoch 54/200\n",
      "4815/4815 [==============================] - 2s 388us/step - loss: 0.0320 - val_loss: 0.0234\n",
      "Epoch 55/200\n",
      "4815/4815 [==============================] - 2s 392us/step - loss: 0.0315 - val_loss: 0.0242\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 56/200\n",
      "4815/4815 [==============================] - 2s 363us/step - loss: 0.0326 - val_loss: 0.0248\n",
      "Epoch 57/200\n",
      "4815/4815 [==============================] - 2s 359us/step - loss: 0.0317 - val_loss: 0.0226\n",
      "Epoch 58/200\n",
      "4815/4815 [==============================] - 2s 355us/step - loss: 0.0321 - val_loss: 0.0242\n",
      "Epoch 59/200\n",
      "4815/4815 [==============================] - 2s 361us/step - loss: 0.0324 - val_loss: 0.0227\n",
      "Epoch 60/200\n",
      "4815/4815 [==============================] - 2s 368us/step - loss: 0.0300 - val_loss: 0.0242\n",
      "Epoch 61/200\n",
      "4815/4815 [==============================] - 2s 369us/step - loss: 0.0307 - val_loss: 0.0231\n",
      "Epoch 62/200\n",
      "4815/4815 [==============================] - 2s 370us/step - loss: 0.0307 - val_loss: 0.0234\n",
      "Epoch 63/200\n",
      "4815/4815 [==============================] - 2s 369us/step - loss: 0.0298 - val_loss: 0.0224\n",
      "Epoch 64/200\n",
      "4815/4815 [==============================] - 2s 382us/step - loss: 0.0286 - val_loss: 0.0223\n",
      "Epoch 65/200\n",
      "4815/4815 [==============================] - 2s 374us/step - loss: 0.0295 - val_loss: 0.0255\n",
      "Epoch 66/200\n",
      "4815/4815 [==============================] - 2s 371us/step - loss: 0.0290 - val_loss: 0.0245\n",
      "Epoch 67/200\n",
      "4815/4815 [==============================] - 2s 372us/step - loss: 0.0279 - val_loss: 0.0232\n",
      "Epoch 68/200\n",
      "4815/4815 [==============================] - 2s 389us/step - loss: 0.0292 - val_loss: 0.0217\n",
      "Epoch 69/200\n",
      "4815/4815 [==============================] - 2s 371us/step - loss: 0.0276 - val_loss: 0.0271\n",
      "Epoch 70/200\n",
      "4815/4815 [==============================] - 2s 373us/step - loss: 0.0288 - val_loss: 0.0231\n",
      "Epoch 71/200\n",
      "4815/4815 [==============================] - 2s 372us/step - loss: 0.0275 - val_loss: 0.0228\n",
      "Epoch 72/200\n",
      "4815/4815 [==============================] - 2s 417us/step - loss: 0.0264 - val_loss: 0.0223\n",
      "Epoch 73/200\n",
      "4815/4815 [==============================] - 2s 370us/step - loss: 0.0271 - val_loss: 0.0217\n",
      "Epoch 74/200\n",
      "4815/4815 [==============================] - 2s 373us/step - loss: 0.0297 - val_loss: 0.0230\n",
      "Epoch 75/200\n",
      "4815/4815 [==============================] - 2s 370us/step - loss: 0.0259 - val_loss: 0.0249\n",
      "Epoch 76/200\n",
      "4815/4815 [==============================] - 2s 373us/step - loss: 0.0278 - val_loss: 0.0244\n",
      "Epoch 77/200\n",
      "4815/4815 [==============================] - 2s 377us/step - loss: 0.0272 - val_loss: 0.0251\n",
      "Epoch 78/200\n",
      "4815/4815 [==============================] - 2s 383us/step - loss: 0.0270 - val_loss: 0.0246\n",
      "Epoch 79/200\n",
      "4815/4815 [==============================] - 2s 373us/step - loss: 0.0260 - val_loss: 0.0228\n",
      "Epoch 80/200\n",
      "4815/4815 [==============================] - 2s 377us/step - loss: 0.0258 - val_loss: 0.0241\n",
      "Epoch 81/200\n",
      "4815/4815 [==============================] - 2s 381us/step - loss: 0.0266 - val_loss: 0.0225\n",
      "Epoch 82/200\n",
      "4815/4815 [==============================] - 2s 373us/step - loss: 0.0255 - val_loss: 0.0219\n",
      "Epoch 83/200\n",
      "4815/4815 [==============================] - 2s 376us/step - loss: 0.0258 - val_loss: 0.0222\n",
      "Epoch 00083: early stopping\n",
      "Test RMSE :  0.14913556757980279\n",
      "Running CV Iteration Num : 5\n",
      "Train on 4816 samples, validate on 1203 samples\n",
      "Epoch 1/200\n",
      "4816/4816 [==============================] - 4s 821us/step - loss: 0.6196 - val_loss: 0.0681\n",
      "Epoch 2/200\n",
      "4816/4816 [==============================] - 2s 383us/step - loss: 0.1881 - val_loss: 0.0527\n",
      "Epoch 3/200\n",
      "4816/4816 [==============================] - 2s 384us/step - loss: 0.1355 - val_loss: 0.0454\n",
      "Epoch 4/200\n",
      "4816/4816 [==============================] - 2s 384us/step - loss: 0.1189 - val_loss: 0.0398\n",
      "Epoch 5/200\n",
      "4816/4816 [==============================] - 2s 381us/step - loss: 0.1041 - val_loss: 0.0305\n",
      "Epoch 6/200\n",
      "4816/4816 [==============================] - 2s 388us/step - loss: 0.0937 - val_loss: 0.0296\n",
      "Epoch 7/200\n",
      "4816/4816 [==============================] - 2s 379us/step - loss: 0.0925 - val_loss: 0.0505\n",
      "Epoch 8/200\n",
      "4816/4816 [==============================] - 2s 395us/step - loss: 0.0815 - val_loss: 0.0281\n",
      "Epoch 9/200\n",
      "4816/4816 [==============================] - 2s 412us/step - loss: 0.0789 - val_loss: 0.0236\n",
      "Epoch 10/200\n",
      "4816/4816 [==============================] - 2s 439us/step - loss: 0.0806 - val_loss: 0.0261\n",
      "Epoch 11/200\n",
      "4816/4816 [==============================] - 2s 398us/step - loss: 0.0760 - val_loss: 0.0236\n",
      "Epoch 12/200\n",
      "4816/4816 [==============================] - 2s 384us/step - loss: 0.0749 - val_loss: 0.0262\n",
      "Epoch 13/200\n",
      "4816/4816 [==============================] - 2s 387us/step - loss: 0.0724 - val_loss: 0.0300\n",
      "Epoch 14/200\n",
      "4816/4816 [==============================] - 2s 386us/step - loss: 0.0688 - val_loss: 0.0229\n",
      "Epoch 15/200\n",
      "4816/4816 [==============================] - 2s 387us/step - loss: 0.0680 - val_loss: 0.0269\n",
      "Epoch 16/200\n",
      "4816/4816 [==============================] - 2s 391us/step - loss: 0.0610 - val_loss: 0.0226\n",
      "Epoch 17/200\n",
      "4816/4816 [==============================] - 2s 393us/step - loss: 0.0628 - val_loss: 0.0224\n",
      "Epoch 18/200\n",
      "4816/4816 [==============================] - 2s 373us/step - loss: 0.0601 - val_loss: 0.0255\n",
      "Epoch 19/200\n",
      "4816/4816 [==============================] - 2s 422us/step - loss: 0.0580 - val_loss: 0.0314\n",
      "Epoch 20/200\n",
      "4816/4816 [==============================] - 2s 387us/step - loss: 0.0583 - val_loss: 0.0229\n",
      "Epoch 21/200\n",
      "4816/4816 [==============================] - 2s 388us/step - loss: 0.0563 - val_loss: 0.0221\n",
      "Epoch 22/200\n",
      "4816/4816 [==============================] - 2s 388us/step - loss: 0.0543 - val_loss: 0.0199\n",
      "Epoch 23/200\n",
      "4816/4816 [==============================] - 2s 386us/step - loss: 0.0539 - val_loss: 0.0209\n",
      "Epoch 24/200\n",
      "4816/4816 [==============================] - 2s 386us/step - loss: 0.0533 - val_loss: 0.0239\n",
      "Epoch 25/200\n",
      "4816/4816 [==============================] - 2s 390us/step - loss: 0.0518 - val_loss: 0.0241\n",
      "Epoch 26/200\n",
      "4816/4816 [==============================] - 2s 380us/step - loss: 0.0498 - val_loss: 0.0213\n",
      "Epoch 27/200\n",
      "4816/4816 [==============================] - 2s 392us/step - loss: 0.0490 - val_loss: 0.0236\n",
      "Epoch 28/200\n",
      "4816/4816 [==============================] - 2s 407us/step - loss: 0.0477 - val_loss: 0.0199\n",
      "Epoch 29/200\n",
      "4816/4816 [==============================] - 2s 416us/step - loss: 0.0469 - val_loss: 0.0212\n",
      "Epoch 30/200\n",
      "4816/4816 [==============================] - 2s 396us/step - loss: 0.0460 - val_loss: 0.0231\n",
      "Epoch 31/200\n",
      "4816/4816 [==============================] - 2s 380us/step - loss: 0.0467 - val_loss: 0.0198\n",
      "Epoch 32/200\n",
      "4816/4816 [==============================] - 2s 388us/step - loss: 0.0473 - val_loss: 0.0196\n",
      "Epoch 33/200\n",
      "4816/4816 [==============================] - 2s 385us/step - loss: 0.0415 - val_loss: 0.0236\n",
      "Epoch 34/200\n",
      "4816/4816 [==============================] - 2s 412us/step - loss: 0.0446 - val_loss: 0.0220\n",
      "Epoch 35/200\n",
      "4816/4816 [==============================] - 2s 394us/step - loss: 0.0434 - val_loss: 0.0228\n",
      "Epoch 36/200\n",
      "4816/4816 [==============================] - 2s 395us/step - loss: 0.0401 - val_loss: 0.0196\n",
      "Epoch 37/200\n",
      "4816/4816 [==============================] - 2s 385us/step - loss: 0.0419 - val_loss: 0.0215\n",
      "Epoch 38/200\n",
      "4816/4816 [==============================] - 2s 395us/step - loss: 0.0403 - val_loss: 0.0236\n",
      "Epoch 39/200\n",
      "4816/4816 [==============================] - 2s 388us/step - loss: 0.0394 - val_loss: 0.0207\n",
      "Epoch 40/200\n",
      "4816/4816 [==============================] - 2s 386us/step - loss: 0.0386 - val_loss: 0.0222\n",
      "Epoch 41/200\n",
      "4816/4816 [==============================] - 2s 398us/step - loss: 0.0401 - val_loss: 0.0234\n",
      "Epoch 42/200\n",
      "4816/4816 [==============================] - 2s 391us/step - loss: 0.0397 - val_loss: 0.0235\n",
      "Epoch 00042: early stopping\n",
      "Test RMSE :  0.1534053985557962\n",
      "CV RMSE :  0.15639046727479403\n"
     ]
    }
   ],
   "source": [
    "IterationNum = 1\n",
    "for train_index, test_index in kf.split(normed_x_train02):\n",
    "    print(\"Running CV Iteration Num :\", IterationNum)\n",
    "    MOD_DATA_2_TRAIN, MOD_DATA_2_TEST = train01.iloc[train_index], train01.iloc[test_index]\n",
    "    X_TRAIN, X_TEST = normed_x_train02.iloc[train_index], normed_x_train02.iloc[test_index]\n",
    "    Y_TRAIN, Y_TEST = y_train02[train_index], y_train02[test_index]\n",
    "    \n",
    "    model = nn_model()\n",
    "    es = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=10)\n",
    "    fin_model = model.fit(X_TRAIN, Y_TRAIN, validation_data = (X_TEST, Y_TEST), epochs = 200, batch_size = 32, verbose = 1, callbacks=[es])\n",
    "    \n",
    "    MOD_DATA_2_TEST['Predicted_Model_Value'] = model.predict(X_TEST)[:,0]\n",
    "    if(IterationNum == 1):\n",
    "        CV_SCORED_DATA = MOD_DATA_2_TEST.copy(deep=True)\n",
    "        CV_SCORED_DATA.reset_index(drop = True, inplace = True)\n",
    "        test_scored = model.predict(normed_x_sub2)[:,0]\n",
    "    else:\n",
    "        CV_SCORED_DATA = pd.concat([CV_SCORED_DATA,MOD_DATA_2_TEST])\n",
    "        CV_SCORED_DATA.reset_index(drop = True, inplace = True)\n",
    "        test_scored = test_scored + model.predict(normed_x_sub2)[:,0]\n",
    "                    \n",
    "    IterationNum = IterationNum + 1\n",
    "    \n",
    "    print(\"Test RMSE : \",sqrt(mean_squared_error(MOD_DATA_2_TEST[\"Price\"], MOD_DATA_2_TEST['Predicted_Model_Value'])))\n",
    "    \n",
    "print(\"CV RMSE : \",sqrt(mean_squared_error(CV_SCORED_DATA[\"Price\"], CV_SCORED_DATA['Predicted_Model_Value'])))\n",
    "#CV RMSE :  0.15639046727479403\n",
    "#LB: 0.9387"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DescribeResult(nobs=1234, minmax=(0.73964167, 4.1613693), mean=1.9749265, variance=0.490959, skewness=0.7544870376586914, kurtosis=0.08592567944426888)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from scipy import stats\n",
    "test_scored2 = test_scored / 5.0\n",
    "stats.describe(test_scored2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2.607597</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2.792425</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>16.253904</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4.002155</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4.369020</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Price\n",
       "0   2.607597\n",
       "1   2.792425\n",
       "2  16.253904\n",
       "3   4.002155\n",
       "4   4.369020"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_scored3 = pd.DataFrame({'Price' : np.exp(test_scored2)-1})\n",
    "test_scored3.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "CV_SCORED_DATA.to_csv(\"C:\\\\Kaggle\\\\Cars\\\\CV_Scored\\\\20190716_Keras01copy_CVTRAIN_DS.csv\",\n",
    "                      index = False)\n",
    "test_scored3.to_csv(\"C:\\\\Kaggle\\\\Cars\\\\Submission\\\\20190716_Keras01copy_TEST_DS.csv\",\n",
    "                    index = False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
